import os
import app
import json
import subprocess as sp

DATA_LOCATION = 'courseScraper/offered.json'

def setup_current_klasses():
    if os.path.exists(DATA_LOCATION):
        os.remove(DATA_LOCATION)

    p = sp.Popen('cd courseScraper && scrapy crawl new-course-spider -o offered.json -t json', stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE, shell=True)
    out = p.communicate()
    # this is to show the output of the scraping
    print out[1]

    json_data = open(DATA_LOCATION)
    courses = json.load(json_data)
    db = app.connect_db()

    for c in courses:
        a = db.execute('UPDATE klasses SET offered = 1 WHERE department = "%s" AND course_num = "%s"' % (c['department'], c['course_num']))
        db.commit()

    db.close()

def main():
    try:
        import scrapy
        import twisted
        setup_current_klasses()
    except ImportError, e:
        print 'No Scrapy'

main()
if __name__ == "__main__":
    app.app.run(debug = True)
